#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dirent.h>
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <set>
#include <map>
#include <db.h>
#include <sys/time.h>
#include <unistd.h>
#include <assert.h>
#include <signal.h>

using namespace std;
const bool incflag = true; //索引更新方式：增量更新或简单重新构造 true为增量更新，false为重新构造
const int interval = 3600*24; //检查更新的时间间隔
const int MAX_INS_NUM = 2; //新增文件最大个数，若超过这个数字，更新合并数据库
const int MAX_DEL_NUM = 2; //插入文件最大个数，若超过这个数字，更新合并数据库
const int MAX_FILE_NUM = 10000; //输入文件最大数目
string strfile[MAX_FILE_NUM];  //存放输入文件名
int fileind; //记录原有文件个数
string updatestrfile[MAX_FILE_NUM];
int updatefileind; //记录更新的文件个数
string insstrfile[MAX_FILE_NUM];
int main_upd[MAX_FILE_NUM]; //表示strfile到updatefile的映射
int ins_upd[MAX_FILE_NUM]; //表示insfile到updatefile的映射
int insfileind; //记录新增文件的个数
int foreinsfileind; //记录上次新增文件的个数
int delfileind; //记录删除文件个数
DB *dbp;      //数据库
DBT key, data; 
DBC *cursorp;

typedef struct pos_num{ //每个文件的分词数据项
        int pos; //第一次出现的行数
        int tf;  //关键字出现的次数
}POS_NUM;

typedef struct keynum //存入数据库中的数据项
{
        int docid; //文件id
        int tf; //关键字出现次数
        int pos; //关键字第一次出现的行数
}KEYNUM;

map<string,POS_NUM> mapkey;   //保存每个文件的分词结果
map<string,POS_NUM> mapkey2; //暂时存放正向分词结果
map<string,POS_NUM> mapkey3; //暂时存放逆向分词结果
int len2;//存放正向分词分词个数
int len3;//存放逆向分词分词个数
int cnt2;//对正向分词中单个字计数
int cnt3;//对逆向分词中单个字计数
set<string> keyst; //存放字典的set
set<int> deldocid; //存放待删除的docid

const char *mainfilename = "../filename/mainfilename.txt";
const char *delfilename = "../filename/delfilename.txt";
const char *insfilename = "../filename/insfilename.txt";
const char *sourcedir = "../source/";
const char *dicfile = "../dict/words_gbk.dic";
const char *mainindexdb = "../db/mainindex.db";
const char *auxindexdb = "../db/auxindex.db";


void cinfile(const char *dir)  //对文件目录进行read，获取所有文件名
{
        ofstream outfile(mainfilename,ofstream::out);
	DIR *dirptr = NULL;
	struct dirent * entry;
        if((dirptr = opendir(dir))==NULL){
		printf("opendir error!");
		return ;
	}
	fileind = 0;
	char dirtmp[200];
	while((entry = readdir(dirptr)) != NULL){
		if(entry->d_name[0]!='.'){
			strcpy(dirtmp,dir);
			strcat(dirtmp,entry->d_name);
			strfile[fileind] = string(dirtmp);
			outfile<<strfile[fileind++]<<endl;
			
		}
	}
        outfile.close();
	closedir(dirptr);
}

void updatefile(const char *dir)  //再次读目录
{
	DIR *dirptr = NULL;
	struct dirent * entry;
        if((dirptr = opendir(dir))==NULL){
		printf("opendir error!");
		return ;
	}
	char dirtmp[200];
	updatefileind = 0;
	while((entry = readdir(dirptr)) != NULL){
		if(entry->d_name[0]!='.'){
			strcpy(dirtmp,dir);
			strcat(dirtmp,entry->d_name);
			updatestrfile[updatefileind++] = string(dirtmp);		
		}
	}
	closedir(dirptr);
}

void check()  //检查更新
{
       ofstream outfile(delfilename,ofstream::out);
       deldocid.clear();
       for(int i=0;i<fileind;i++){
                bool flag = 0;
                for(int j=0;j<updatefileind;j++){
                        if(strfile[i]==updatestrfile[j]){
                                flag = 1;
                                main_upd[i] = j;//保存原有文件docid与更新后的docid的映射
                                break;
                        }
                }
                if(!flag){
                        outfile<<strfile[i]<<endl;
                        deldocid.insert(i); //保存删除文件名
                }
       }
       outfile.close();
       FILE *fout = fopen(insfilename,"a");
       if(fout==NULL){
                puts("open infile error");
                return;
       }
       foreinsfileind = insfileind; //考虑不超过MAX_INS_FILE的增量新增文件
       for(int i=0;i<updatefileind;i++){
                bool flag = 0;
                for(int j=0;j<fileind;j++){
                        if(updatestrfile[i]==strfile[j]){
                                flag = 1;
                                break;
                        }
                }
                if(!flag){
                        
                        fputs(updatestrfile[i].c_str(),fout);
                        ins_upd[insfileind]=i; //保存新增文件docid与更新后的docid的映射
                        insstrfile[insfileind++] = updatestrfile[i];
                }
       }
       fclose(fout);
}

void builddict()  //创建字典
{
        ifstream infile(dicfile,ifstream::in);
        if(!infile.is_open()){
		printf("open inputfile error!\n");
		return ;
	}
	int num;
	string keys;
	while(infile>>num>>keys){
	       keyst.insert(keys);
        }
        infile.close();
}

bool find(string str)  //查找关键词是否存在
{
	if(keyst.find(str)!=keyst.end())
		return true;
	else
		return false;
}

bool stoplisten(char ch)  //英文停用词表
{
        return ch=='.'||ch==' '||ch==';'||ch=='\t';
}

void incnum(string str,int pos) //将关键词添加倒map中，并记录关键词第一次出现的位置
{
       map<string,POS_NUM>::iterator iter =  mapkey.find(str);
       if(iter==mapkey.end()){
             POS_NUM tmp;
             tmp.tf = 0;
             tmp.pos = pos;
             mapkey.insert(pair<string,POS_NUM>(str,tmp));
       }
       ++(mapkey[str].tf);
}

string strtolower(string str)
{
       for(int i=0;i<str.length();i++){
            str[i] = tolower(str[i]);
       }
       return str;
}

void segmenten(string enstr,int pos)  //将英文串分词
{
        int ind = 0;
        while(enstr[ind]&&stoplisten(enstr[ind])) ind++;
        if(!enstr[ind])
                return;
        bool flag = false;
        for(int i=ind;enstr[i];i++){
                if(!flag&&stoplisten(enstr[i])){
                       flag=true;
                       incnum(strtolower(enstr.substr(ind,i-ind)),pos);
                       ind = i;
                }
                else if(flag&&!stoplisten(enstr[i])){
                       flag = false;
                       ind = i;
                }
        }
        int len = enstr.length();
        incnum(strtolower(enstr.substr(ind,len-ind)),pos);
}


void incnum2(string str,int pos)//将逆向分词结果暂时保存
{
        map<string,POS_NUM>::iterator iter =  mapkey2.find(str);
        if(iter==mapkey2.end()){
             POS_NUM tmp;
             tmp.tf = 0;
             tmp.pos = pos;
             mapkey2.insert(pair<string,POS_NUM>(str,tmp));
        }
        ++len2;
        if(str.length()==1)
              ++cnt2;
        ++(mapkey2[str].tf);
}

void incnum3(string str,int pos) //将逆向分词结果暂时保存
{
        map<string,POS_NUM>::iterator iter =  mapkey3.find(str);
        if(iter==mapkey3.end()){
             POS_NUM tmp;
             tmp.tf = 0;
             tmp.pos = pos;
             mapkey3.insert(pair<string,POS_NUM>(str,tmp));
        }
        ++len3;
        if(str.length()==1)
             ++cnt3;
        ++(mapkey3[str].tf);
}

void segmentch(string enstr,int pos,int flag) //将中文串分词
{      
        int len = enstr.length();
        int clen;
        string st;
        if(flag){   // 逆向最大匹配
                while(!enstr.empty()){
                  st = enstr;
                  clen = len;
                  while(clen>2&&!find(st)){
                    clen -=1;  //若在字典找不到对应的关键字，则一个个输出
                    st = enstr.substr(len-clen,clen);
                  }
                  if(clen>0){
                    incnum2(st,pos);
                    enstr = enstr.substr(0,len-clen);
                  }     
                  len = enstr.length();          
               }
        }
        else{   //正向最大匹配
               while(!enstr.empty()){
                   st = enstr;
                   clen = len;
                   while(clen>2&&!find(st)){
                       clen -= 1;
                       st = enstr.substr(0,clen);
                   }
                   if(clen>0){
                       incnum3(st,pos);
                       enstr = enstr.substr(clen,len-clen);
                   }
                   len = enstr.length();
               }
        }
}

bool comp()  //比对正向最大分词和逆向最大分词，若相同return true,否则return false
{
       if(mapkey2.size()!=mapkey3.size())
              return false;
       for(map<string,POS_NUM>::iterator iter2 = mapkey2.begin(),iter3 = mapkey3.begin();iter2 != mapkey2.end() && iter3 != mapkey3.end(); iter2++,iter3++){
              if(iter2->first == iter3->first && (iter2->second).pos == (iter3->second).pos && (iter2->second).tf == (iter3->second).tf){
                    continue;
              }
              else
                    return false;
       }
       return true;
}

void segmentboth(string enstr,int pos) //进行逆向分词和正向分词的主控程序，若两者不相同，则比较两者分词个数，考虑分词个数少的那种，若相同，则考虑两种分词中单字的个数最少的，若仍相同，选择正向分词
{
        cnt2 = cnt3 = 0;
        len2 = len3 = 0;
        mapkey2.clear();
        mapkey3.clear();
        segmentch(enstr,pos,0);
        segmentch(enstr,pos,1);
        int flag = 1;
        if(comp()){
              flag = 1;
              
        }
        else{
              if(len2 < len3){
                     flag = 1;
              }
              else if(len2 > len3){
                     flag = -1;
              }
              else{
                     if(cnt2<cnt3)
                          flag = 1;
                     else if(cnt2>cnt3)
                          flag = -1;
                     else 
                          flag = 1;
              }
        }
        if(flag == 1){  //将正向分词结果并入
              for(map<string,POS_NUM>::iterator iter = mapkey2.begin();iter != mapkey2.end();iter++){
                    map<string,POS_NUM>::iterator iter0 =  mapkey.find(iter->first);
                    if(iter0==mapkey.end()){
                         POS_NUM tmp;
                         tmp.tf = (iter->second).tf;
                         tmp.pos = (iter->second).pos;
                         mapkey.insert(pair<string,POS_NUM>(iter->first,tmp));
                    }
                    else
                         (iter0->second).tf += (iter->second).tf;
              }
        }
        else{  //将逆向分词结果并入
              for(map<string,POS_NUM>::iterator iter = mapkey3.begin();iter != mapkey3.end();iter++){
                    map<string,POS_NUM>::iterator iter0 =  mapkey.find(iter->first);
                    if(iter0==mapkey.end()){
                         POS_NUM tmp;
                         tmp.tf = (iter->second).tf;
                         tmp.pos = (iter->second).pos;
                         mapkey.insert(pair<string,POS_NUM>(iter->first,tmp));
                    }
                    else
                         (iter0->second).tf += (iter->second).tf;
              }
        }
}


void segmentstring(string st,int pos) //st为输入字符串，最大逆序匹配 ：由于设定了关键字最大长度，对于中英文混合分词处理起来有一定的麻烦，这里只是将分隔的英文分开，最后再将中文分词
{
	string st2;
	int len = st.length();
	int clen ;
	while(!st.empty()){
	       clen = len;
	       st2 = st.substr(len-clen,len);
	       bool flag = (st2[0]>0)?1:0;
	       int ind = 0;
	       for(int i =0;i<clen;i++){
	             if(flag&&st2[i]<0){
	                   flag = 0;
	                 //  cout<<st2.substr(ind,i-ind)<<'#'<<endl;
	                   segmenten(st2.substr(ind,i-ind),pos);
	                   ind = i;
                     }
                     else if(!flag&&st2[i]>0){
                           flag =1 ;
                         //  cout<<st2.substr(ind,i-ind)<<'*'<<endl;
                           segmentboth(st2.substr(ind,i-ind),pos);
                           ind = i;
                     }
               }
	       if(st2[ind]>0){
	            // cout<<st2.substr(ind,clen-ind)<<'#'<<endl;
	             segmenten(st2.substr(ind,clen-ind),pos);
               }
               else{
                     //cout<<st2.substr(ind,clen-ind)<<'*'<<endl;
                     segmentboth(st2.substr(ind,clen-ind),pos);
               }
               st = st.substr(0,len-clen);
               len = st.length();
        }
}

void print_error(int ret)
{
	if(ret != 0)
		printf("ERROR: %s\n",db_strerror(ret));
}


void segmentall(int fore,int ind,string sstrfile[]) //对所有文件进行分词处理
{
        for(int i = 0;i<ind;i++){
               ifstream infile(sstrfile[i].c_str(),ifstream::in);
               mapkey.clear();
               if(!infile.is_open()){
		    printf("open inputfile error!\n");
		    continue;
	       }
	       string strtmp;
	       int pos = 0;
	       while(getline(infile,strtmp,'\n')){ //取出其中一行进行分词处理
	             segmentstring(strtmp,++pos);
               }
               infile.close();
               for(map<string,POS_NUM>::iterator iter = mapkey.begin();iter!=mapkey.end();iter++){
                     key.size = (iter->first).length()+1;
                     key.data = (char *)((iter->first).c_str());
                     data.size = sizeof(KEYNUM);
                     KEYNUM tmp;
                     tmp.docid = i;
                     tmp.tf = (iter->second).tf;
                     tmp.pos = (iter->second).pos;
                     data.data = &tmp;
                     int ret = cursorp->put(cursorp,&key,&data,DB_KEYFIRST);
	             print_error(ret);
               }
        }
        if(cursorp!=NULL)
               cursorp->c_close(cursorp);
        if(dbp != NULL)
    	       dbp->close(dbp, 0); 
}

int dbcmp(DB *dbp,const DBT *data1,const DBT *data2)
{
         return (((KEYNUM *)(data2->data))->tf)-(((KEYNUM *)(data1->data))->tf);
}

//DBT 初始化
void init_DBT(DBT * key, DBT * data)
{
	memset(key, 0, sizeof(DBT));
	memset(data, 0, sizeof(DBT));
}


void init_db(const char *dbname) //设置berkeley DB ，并打开数据库
{
        int ret;
        ret = db_create(&dbp, NULL, 0);
	print_error(ret);
	ret = dbp->set_flags(dbp,DB_DUPSORT); 
	print_error(ret);
	//ret = dbp->set_dup_compare(dbpp,dbcmp);
	//print_error(ret);
	ret = dbp->set_bt_compress(dbp, NULL, NULL);
        print_error(ret);
	ret = dbp->open(dbp, NULL, dbname, NULL, DB_BTREE, DB_CREATE, 0); 
	print_error(ret);
	init_DBT(&key,&data);
	ret = dbp->cursor(dbp,NULL,&cursorp,0);
	print_error(ret);
}

void createmainindex() //创建主索引数据库
{
        builddict(); 
        cinfile(sourcedir);
        init_db(mainindexdb);
        segmentall(0,fileind,strfile);
}

void updateindex1(int signo)  //增量更新数据库
{
        updatefile(sourcedir); //重新读目录
        check(); //查找文件更新
        init_db(auxindexdb);
        segmentall(foreinsfileind,insfileind,insstrfile); //将新增的文件分词存入附加索引数据库
        cout<<foreinsfileind<<insfileind<<endl;
        if(insfileind>MAX_INS_NUM||deldocid.size()>MAX_DEL_NUM){ //当删除的文件个数或增加的文件个数超过一定数量，将附加索引数据库并入主索引数据库，并将附加索引数据库清空
                 ofstream outfile(mainfilename,ofstream::out);
                 fileind = updatefileind;
                 for(int i=0;i<updatefileind;i++){ //将原有文件替换为更新后的文件
                     outfile<<updatestrfile[i]<<endl;
                     strfile[i]=updatestrfile[i];
                 }
                 outfile.close(); //清空新增文件
                 ofstream outfile2(insfilename,ofstream::out);
                 outfile2.close(); //清空删除文件
                 ofstream outfile3(delfilename,ofstream::out);
                 outfile3.close();
                 
                 DB *dbpp;
                 DBC *cursorpp;
                 int ret;
                 ret = db_create(&dbpp, NULL, 0);
	         print_error(ret);
	         ret = dbpp->set_flags(dbpp,DB_DUPSORT); 
	         print_error(ret);
	         //ret = dbp->set_dup_compare(dbpp,dbcmp);
	         //print_error(ret);
	         ret = dbpp->set_bt_compress(dbpp, NULL, NULL);
                 print_error(ret);
	         ret = dbpp->open(dbpp, NULL, auxindexdb, NULL, DB_BTREE, DB_CREATE, 0); //打开附加索引数据库
	         print_error(ret);
	         init_DBT(&key,&data);
	         ret = dbpp->cursor(dbpp,NULL,&cursorpp,0);
	         print_error(ret);
	         
                 init_db(mainindexdb); //打开主索引数据库
                 DBC *cursorp2;
                 ret = dbp->cursor(dbp,NULL,&cursorp2,0);//为防止将遍历cursorp改变，再定义另一个cursorp2
                 print_error(ret);
                 ret = cursorp->c_get(cursorp,&key,&data,DB_FIRST);
                 while(ret!=DB_NOTFOUND){ 
                     int tmp = ((KEYNUM *)(data.data))->docid;
                     if(deldocid.find(tmp)!=deldocid.end()) //删除有删除文件的数据项
                           cursorp->del(cursorp,0);
                     else{
                           cursorp->del(cursorp,0);
                           ((KEYNUM *)(data.data))->docid = main_upd[tmp]; //将文件docid更新为最新的
                           ret = cursorp2->put(cursorp2,&key,&data,DB_KEYFIRST);
                           print_error(ret);
                     }
                     ret = cursorp->c_get(cursorp,&key,&data,DB_NEXT);
                 }
                 if(cursorp2!=NULL)
                     cursorp2->c_close(cursorp2);
                 ret = cursorpp->c_get(cursorpp,&key,&data,DB_FIRST);
	         while(ret!=DB_NOTFOUND){  //从附加索引数据库取数据放入主索引数据库
	             int tmpp = ((KEYNUM *)(data.data))->docid;
	             ((KEYNUM *)(data.data))->docid = ins_upd[tmpp];
	             int tmp = cursorp->put(cursorp,&key,&data,DB_KEYFIRST);
                     print_error(tmp);
	             ret = cursorpp->c_get(cursorpp,&key,&data,DB_NEXT);
                 }
                 if(cursorp!=NULL)
                     cursorp->c_close(cursorp);
                 if(dbp != NULL)
    	             dbp->close(dbp, 0); 
    	         if(cursorpp!=NULL)
                     cursorpp->c_close(cursorpp);
                 if(dbpp != NULL){
    	             dbpp->close(dbpp, 0); 
    	         }
    	         DB_ENV *dbenv;
    	         ret = db_create(&dbpp,NULL,0);
    	         print_error(ret);
    	         ret = dbpp->remove(dbpp,auxindexdb,NULL,0);  //删除附加索引数据库
                 print_error(ret);
                 insfileind = 0;
        }
}

void updateindex2(int signo) //重新构造索引
{
        updatefile(sourcedir);
        ofstream outfile(mainfilename,ofstream::out);
        for(int i=0;i<updatefileind;i++){
                strfile[i] = updatestrfile[i];
		outfile<<strfile[i]<<endl;
        }
        outfile.close();
        fileind = updatefileind;
        createmainindex();
}

   // 建立信号处理机制
void init_sigaction(void)
{
     struct sigaction tact;
     if(incflag)
            tact.sa_handler = updateindex1;
     else
            tact.sa_handler = updateindex2;
     tact.sa_handler = updateindex1;
     tact.sa_flags = 0;
     sigemptyset(&tact.sa_mask);
     sigaction(SIGALRM, &tact, NULL);
}
 
void init_time() //设置更新间隔时间
{
     struct itimerval value;
     value.it_value.tv_sec = interval; //以秒为单位
     value.it_value.tv_usec = 0;
     value.it_interval = value.it_value;
     setitimer(ITIMER_REAL, &value, NULL);
}

int main()
{
        createmainindex();
        puts("create mainindex ok");
        init_sigaction();
        init_time();
        FILE *pin = popen("./query","r");
        if(pin==NULL){
              puts("popen error");
              exit(1);
        }
        char buf[1024];
        while(fgets(buf,1024,pin)!=NULL){
              puts(buf);
        }
        pclose(pin);
        while(1) ;
        return 0;
}


